<!doctype html>














<!-- `site.alt_lang` can specify a language different from the UI -->
<html lang="en" 
  
>
  <!-- The Head -->

<head>
  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  <meta name="theme-color" media="(prefers-color-scheme: light)" content="#f7f7f7">
  <meta name="theme-color" media="(prefers-color-scheme: dark)" content="#1b1b1e">
  <meta name="apple-mobile-web-app-capable" content="yes">
  <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
  <meta
    name="viewport"
    content="width=device-width, user-scalable=no initial-scale=1, shrink-to-fit=no, viewport-fit=cover"
  >

  

  

  
    <!-- Begin Jekyll SEO tag v2.8.0 -->
<meta name="generator" content="Jekyll v4.3.2" />
<meta property="og:title" content="Linux Heterogeneous memory management (HMM)" />
<meta name="author" content="Chunsheng Luo" />
<meta property="og:locale" content="en" />
<meta name="description" content="一、HMM是什么？ Reference：异构内存管理 Heterogeneous Memory Management (HMM)" />
<meta property="og:description" content="一、HMM是什么？ Reference：异构内存管理 Heterogeneous Memory Management (HMM)" />
<link rel="canonical" href="https://luochenglcs.github.io/posts/kernel-hmm/" />
<meta property="og:url" content="https://luochenglcs.github.io/posts/kernel-hmm/" />
<meta property="og:site_name" content="Cheng Luo" />
<meta property="og:type" content="article" />
<meta property="article:published_time" content="2021-03-04T04:10:00+00:00" />
<meta name="twitter:card" content="summary" />
<meta property="twitter:title" content="Linux Heterogeneous memory management (HMM)" />
<meta name="twitter:site" content="@luochenglcs" />
<meta name="twitter:creator" content="@Chunsheng Luo" />
<script type="application/ld+json">
{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Chunsheng Luo","url":"https://github.com/luochenglcs"},"dateModified":"2023-08-08T07:06:12+00:00","datePublished":"2021-03-04T04:10:00+00:00","description":"一、HMM是什么？ Reference：异构内存管理 Heterogeneous Memory Management (HMM)","headline":"Linux Heterogeneous memory management (HMM)","mainEntityOfPage":{"@type":"WebPage","@id":"https://luochenglcs.github.io/posts/kernel-hmm/"},"url":"https://luochenglcs.github.io/posts/kernel-hmm/"}</script>
<!-- End Jekyll SEO tag -->

  

  <title>Linux Heterogeneous memory management (HMM) | Cheng Luo
  </title>

  <!--
  The Favicons for Web, Android, Microsoft, and iOS (iPhone and iPad) Apps
  Generated by: https://realfavicongenerator.net/
-->



<link rel="apple-touch-icon" sizes="180x180" href="/assets/img/favicons/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/assets/img/favicons/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/assets/img/favicons/favicon-16x16.png">
<link rel="manifest" href="/assets/img/favicons/site.webmanifest">
<link rel="shortcut icon" href="/assets/img/favicons/favicon.ico">
<meta name="apple-mobile-web-app-title" content="Cheng Luo">
<meta name="application-name" content="Cheng Luo">
<meta name="msapplication-TileColor" content="#da532c">
<meta name="msapplication-config" content="/assets/img/favicons/browserconfig.xml">
<meta name="theme-color" content="#ffffff">


  
    
      <link rel="preconnect" href="https://fonts.googleapis.com" >
      <link rel="dns-prefetch" href="https://fonts.googleapis.com" >
    
      <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
      <link rel="dns-prefetch" href="https://fonts.gstatic.com" crossorigin>
    
      <link rel="preconnect" href="https://fonts.googleapis.com" >
      <link rel="dns-prefetch" href="https://fonts.googleapis.com" >
    
      <link rel="preconnect" href="https://cdn.jsdelivr.net" >
      <link rel="dns-prefetch" href="https://cdn.jsdelivr.net" >
    

    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Lato&family=Source+Sans+Pro:wght@400;600;700;900&display=swap">
  

  <!-- GA -->
  

  <!-- Bootstrap -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/css/bootstrap.min.css">

  <!-- Font Awesome -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@6.4.0/css/all.min.css">

  <link rel="stylesheet" href="/assets/css/style.css">

  
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/tocbot@4.21.0/dist/tocbot.min.css">
  

  
    <!-- Manific Popup -->
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/magnific-popup@1.1.0/dist/magnific-popup.min.css">
  

  <!-- JavaScript -->

  
    <!-- Switch the mode between dark and light. -->

<script type="text/javascript">
  class ModeToggle {
    static get MODE_KEY() {
      return 'mode';
    }
    static get MODE_ATTR() {
      return 'data-mode';
    }
    static get DARK_MODE() {
      return 'dark';
    }
    static get LIGHT_MODE() {
      return 'light';
    }
    static get ID() {
      return 'mode-toggle';
    }

    constructor() {
      if (this.hasMode) {
        if (this.isDarkMode) {
          if (!this.isSysDarkPrefer) {
            this.setDark();
          }
        } else {
          if (this.isSysDarkPrefer) {
            this.setLight();
          }
        }
      }

      let self = this;

      /* always follow the system prefers */
      this.sysDarkPrefers.addEventListener('change', () => {
        if (self.hasMode) {
          if (self.isDarkMode) {
            if (!self.isSysDarkPrefer) {
              self.setDark();
            }
          } else {
            if (self.isSysDarkPrefer) {
              self.setLight();
            }
          }

          self.clearMode();
        }

        self.notify();
      });
    } /* constructor() */

    get sysDarkPrefers() {
      return window.matchMedia('(prefers-color-scheme: dark)');
    }

    get isSysDarkPrefer() {
      return this.sysDarkPrefers.matches;
    }

    get isDarkMode() {
      return this.mode === ModeToggle.DARK_MODE;
    }

    get isLightMode() {
      return this.mode === ModeToggle.LIGHT_MODE;
    }

    get hasMode() {
      return this.mode != null;
    }

    get mode() {
      return sessionStorage.getItem(ModeToggle.MODE_KEY);
    }

    /* get the current mode on screen */
    get modeStatus() {
      if (this.isDarkMode || (!this.hasMode && this.isSysDarkPrefer)) {
        return ModeToggle.DARK_MODE;
      } else {
        return ModeToggle.LIGHT_MODE;
      }
    }

    setDark() {
      document.documentElement.setAttribute(ModeToggle.MODE_ATTR, ModeToggle.DARK_MODE);
      sessionStorage.setItem(ModeToggle.MODE_KEY, ModeToggle.DARK_MODE);
    }

    setLight() {
      document.documentElement.setAttribute(ModeToggle.MODE_ATTR, ModeToggle.LIGHT_MODE);
      sessionStorage.setItem(ModeToggle.MODE_KEY, ModeToggle.LIGHT_MODE);
    }

    clearMode() {
      document.documentElement.removeAttribute(ModeToggle.MODE_ATTR);
      sessionStorage.removeItem(ModeToggle.MODE_KEY);
    }

    /* Notify another plugins that the theme mode has changed */
    notify() {
      window.postMessage(
        {
          direction: ModeToggle.ID,
          message: this.modeStatus
        },
        '*'
      );
    }

    flipMode() {
      if (this.hasMode) {
        if (this.isSysDarkPrefer) {
          if (this.isLightMode) {
            this.clearMode();
          } else {
            this.setLight();
          }
        } else {
          if (this.isDarkMode) {
            this.clearMode();
          } else {
            this.setDark();
          }
        }
      } else {
        if (this.isSysDarkPrefer) {
          this.setLight();
        } else {
          this.setDark();
        }
      }

      this.notify();
    } /* flipMode() */
  } /* ModeToggle */

  const modeToggle = new ModeToggle();
</script>

  

  <!-- A placeholder to allow defining custom metadata -->

</head>


  <body>
    <!-- The Side Bar -->

<div id="sidebar" class="d-flex flex-column align-items-end">
  <div class="profile-wrapper">
    <a href="/" id="avatar" class="rounded-circle">
      
        
        <img src="/assets/img/favicons/android-chrome-512x512.png" width="112" height="112" alt="avatar" onerror="this.style.display='none'">
      
    </a>

    <div class="site-title">
      <a href="/">Cheng Luo</a>
    </div>
    <div class="site-subtitle fst-italic">Linux operating system engineer</div>
  </div>
  <!-- .profile-wrapper -->

  <ul class="nav flex-column flex-grow-1 w-100 ps-0">
    <!-- home -->
    <li class="nav-item">
      <a href="/" class="nav-link">
        <i class="fa-fw fas fa-home"></i>
        <span>HOME</span>
      </a>
    </li>
    <!-- the real tabs -->
    
      <li class="nav-item">
        <a href="/categories/" class="nav-link">
          <i class="fa-fw fas fa-stream"></i>
          

          <span>CATEGORIES</span>
        </a>
      </li>
      <!-- .nav-item -->
    
      <li class="nav-item">
        <a href="/tags/" class="nav-link">
          <i class="fa-fw fas fa-tags"></i>
          

          <span>TAGS</span>
        </a>
      </li>
      <!-- .nav-item -->
    
      <li class="nav-item">
        <a href="/archives/" class="nav-link">
          <i class="fa-fw fas fa-archive"></i>
          

          <span>ARCHIVES</span>
        </a>
      </li>
      <!-- .nav-item -->
    
      <li class="nav-item">
        <a href="/about/" class="nav-link">
          <i class="fa-fw fas fa-info-circle"></i>
          

          <span>ABOUT</span>
        </a>
      </li>
      <!-- .nav-item -->
    
  </ul>
  <!-- ul.nav.flex-column -->

  <div class="sidebar-bottom d-flex flex-wrap  align-items-center w-100">
    
      <button class="mode-toggle btn" aria-label="Switch Mode">
        <i class="fas fa-adjust"></i>
      </button>

      
        <span class="icon-border"></span>
      
    

    
      

      
        <a
          href="https://github.com/luochenglcs"
          aria-label="github"
          

          
            target="_blank"
            
          

          

          
            rel="noopener noreferrer"
          
        >
          <i class="fab fa-github"></i>
        </a>
      
    
      

      
    
      

      
        <a
          href="javascript:location.href = 'mailto:' + ['luochunsheng','ustc.edu'].join('@')"
          aria-label="email"
          

          

          

          
        >
          <i class="fas fa-envelope"></i>
        </a>
      
    
  </div>
  <!-- .sidebar-bottom -->
</div>
<!-- #sidebar -->


    <div id="main-wrapper" class="d-flex justify-content-center">
      <div id="main" class="container px-xxl-5">
        <!-- The Top Bar -->

<div id="topbar-wrapper">
  <div
    id="topbar"
    class="container d-flex align-items-center justify-content-between h-100"
  >
    <span id="breadcrumb">
      

      
        
          
            <span>
              <a href="/">
                Home
              </a>
            </span>

          
        
          
        
          
            
              <span>Linux Heterogeneous memory management (HMM)</span>
            

          
        
      
    </span>
    <!-- endof #breadcrumb -->

    <i id="sidebar-trigger" class="fas fa-bars fa-fw"></i>

    <div id="topbar-title">
      Post
    </div>

    <i id="search-trigger" class="fas fa-search fa-fw"></i>
    <span id="search-wrapper" class="align-items-center">
      <i class="fas fa-search fa-fw"></i>
      <input
        class="form-control"
        id="search-input"
        type="search"
        aria-label="search"
        autocomplete="off"
        placeholder="Search..."
      >
    </span>
    <span id="search-cancel">Cancel</span>
  </div>
</div>

        











<div class="row">
  <!-- core -->
  <div id="core-wrapper" class="col-12 col-lg-11 col-xl-9 pe-xl-4">
    

    <div class="post px-1 px-md-2">
      

      
        
      
        <!-- Refactor the HTML structure -->



<!--
  In order to allow a wide table to scroll horizontally,
  we suround the markdown table with `<div class="table-wrapper">` and `</div>`
-->



<!--
  Fixed kramdown code highlight rendering:
  https://github.com/penibelst/jekyll-compress-html/issues/101
  https://github.com/penibelst/jekyll-compress-html/issues/71#issuecomment-188144901
-->



<!-- Change the icon of checkbox -->


<!-- images -->



  
  

  <!-- CDN URL -->
  

  <!-- Add image path -->
  

  
    
      
      
    

    
    

    

    
    

    
    
    
    

    
      

      
      
      

      
    
      

      
      
      

      
    

    <!-- take out classes -->
    

    

    

    

    <!-- lazy-load images <https://github.com/aFarkas/lazysizes#readme> -->
    
    

    <!-- add image placeholder -->
    
      
    

    <!-- Bypass the HTML-proofer test -->
    

    
      <!-- make sure the `<img>` is wrapped by `<a>` -->
      

      
        <!-- create the image wrapper -->
        
      
    

    <!-- combine -->
    

  
    

    
    

    

    
    

    
    
    
    

    
      

      
      
      

      
    
      

      
      
      

      
    

    <!-- take out classes -->
    

    

    

    

    <!-- lazy-load images <https://github.com/aFarkas/lazysizes#readme> -->
    
    

    <!-- add image placeholder -->
    
      
    

    <!-- Bypass the HTML-proofer test -->
    

    
      <!-- make sure the `<img>` is wrapped by `<a>` -->
      

      
        <!-- create the image wrapper -->
        
      
    

    <!-- combine -->
    

  
    

    
    

    

    
    

    
    
    
    

    
      

      
      
      

      
    
      

      
      
      

      
    

    <!-- take out classes -->
    

    

    

    

    <!-- lazy-load images <https://github.com/aFarkas/lazysizes#readme> -->
    
    

    <!-- add image placeholder -->
    
      
    

    <!-- Bypass the HTML-proofer test -->
    

    
      <!-- make sure the `<img>` is wrapped by `<a>` -->
      

      
        <!-- create the image wrapper -->
        
      
    

    <!-- combine -->
    

  
    

    
    

    

    
    

    
    
    
    

    
      

      
      
      

      
    
      

      
      
      

      
    

    <!-- take out classes -->
    

    

    

    

    <!-- lazy-load images <https://github.com/aFarkas/lazysizes#readme> -->
    
    

    <!-- add image placeholder -->
    
      
    

    <!-- Bypass the HTML-proofer test -->
    

    
      <!-- make sure the `<img>` is wrapped by `<a>` -->
      

      
        <!-- create the image wrapper -->
        
      
    

    <!-- combine -->
    

  
    

    
    

    

    
    

    
    
    
    

    
      

      
      
      

      
    
      

      
      
      

      
    

    <!-- take out classes -->
    

    

    

    

    <!-- lazy-load images <https://github.com/aFarkas/lazysizes#readme> -->
    
    

    <!-- add image placeholder -->
    
      
    

    <!-- Bypass the HTML-proofer test -->
    

    
      <!-- make sure the `<img>` is wrapped by `<a>` -->
      

      
        <!-- create the image wrapper -->
        
      
    

    <!-- combine -->
    

  
    

    
    

    

    
    

    
    
    
    

    
      

      
      
      

      
    
      

      
      
      

      
    

    <!-- take out classes -->
    

    

    

    

    <!-- lazy-load images <https://github.com/aFarkas/lazysizes#readme> -->
    
    

    <!-- add image placeholder -->
    
      
    

    <!-- Bypass the HTML-proofer test -->
    

    
      <!-- make sure the `<img>` is wrapped by `<a>` -->
      

      
        <!-- create the image wrapper -->
        
      
    

    <!-- combine -->
    

  
    

    
    

    

    
    

    
    
    
    

    
      

      
      
      

      
    
      

      
      
      

      
    

    <!-- take out classes -->
    

    

    

    

    <!-- lazy-load images <https://github.com/aFarkas/lazysizes#readme> -->
    
    

    <!-- add image placeholder -->
    
      
    

    <!-- Bypass the HTML-proofer test -->
    

    
      <!-- make sure the `<img>` is wrapped by `<a>` -->
      

      
        <!-- create the image wrapper -->
        
      
    

    <!-- combine -->
    

  

  



<!-- Add header for code snippets -->



<!-- Create heading anchors -->





  
  

  

  
  

  
    
    

    
      
        
        
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    

    

  

  
  

  
    
    

    
      
        
        
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    
      

      
      

      
      
      

      

    

    

  

  
  

  




<!-- return -->




<h1 data-toc-skip>Linux Heterogeneous memory management (HMM)</h1>

<div class="post-meta text-muted">
    <!-- published date -->
    <span>
      Posted
      <!--
  Date format snippet
  See: ${JS_ROOT}/utils/locale-dateime.js
-->





<em
  class=""
  data-ts="1614831000"
  data-df="ll"
  
    data-bs-toggle="tooltip" data-bs-placement="bottom"
  
>
  Mar  4, 2021
</em>

    </span>

    <!-- lastmod date -->
    
    <span>
      Updated
      <!--
  Date format snippet
  See: ${JS_ROOT}/utils/locale-dateime.js
-->





<em
  class=""
  data-ts="1691478372"
  data-df="ll"
  
    data-bs-toggle="tooltip" data-bs-placement="bottom"
  
>
  Aug  8, 2023
</em>

    </span>
    

  

  <div class="d-flex justify-content-between">
    <!-- author(s) -->
    <span>
      

      By

      <em>
      
        
          <a href="https://github.com/luochenglcs">Chunsheng Luo</a>
          
        
      
      </em>
    </span>

    <div>
      <!-- read time -->
      <!-- Calculate the post's reading time, and display the word count in tooltip -->



<!-- words per minute -->










<!-- return element -->
<span
  class="readtime"
  data-bs-toggle="tooltip"
  data-bs-placement="bottom"
  title="3969 words"
>
  <em>22 min</em> read</span>

    </div>

  </div> <!-- .d-flex -->

</div> <!-- .post-meta -->
<div class="post-content">
  <h3 id="一hmm是什么"><span class="me-2">一、HMM是什么？</span><a href="#一hmm是什么" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h3>
<p>Reference：<a href="https://blog.csdn.net/Rong_Toa/article/details/117910321">异构内存管理 Heterogeneous Memory Management (HMM)</a></p>

<p>翻译于：https://elixir.bootlin.com/linux/v5.5-rc2/source/Documentation/vm/hmm.rst</p>

<p>提供基础设施和帮助程序以将非常规内存（设备内存，如板上 GPU 内存）集成到常规内核路径中，其基石是此类内存的专用结构页面（请参阅本文档的第 5 至 7 节）。</p>

<p>HMM 还为 SVM（共享虚拟内存）提供了可选的帮助程序，即允许设备透明地访问与 CPU 一致的程序地址，这意味着 CPU 上的任何有效指针也是该设备的有效指针。这对于简化高级异构计算的使用变得必不可少，其中 GPU、DSP 或 FPGA 用于代表进程执行各种计算。</p>

<p>本文档分为以下部分：在第一部分中，我揭示了与使用特定于设备的内存分配器相关的问题。在第二部分中，我揭示了许多平台固有的硬件限制。第三部分概述了 HMM 设计。第四部分解释了 CPU 页表镜像的工作原理以及 HMM 在这种情况下的目的。第五部分处理内核中如何表示设备内存。最后，最后一节介绍了一个新的迁移助手，它允许利用设备 DMA 引擎。</p>

<h4 id="1-特定设备内存分配器"><span class="me-2">1 特定设备内存分配器</span><a href="#1-特定设备内存分配器" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>
<p><strong><a href="https://www.kernel.org/doc/html/latest/vm/hmm.html#id1">使用特定于设备的内存分配器的问题</a></strong></p>

<p>设备有大量的板载内存，历来都是专有驱动的API管理，使得驱动程序的内存使用与常规应用内存使用断开，存在分裂。</p>

<p>1、大型程序需要依赖大量的库，使得程序维护复杂</p>

<p>2、复杂数据，例如：复杂数据集（列表、树……），在驱动与应用之间copy的话，由于重复的数据集和地址，这很容易出错并且程序更难调试。</p>

<p>3、由于各种内存副本，大型项目会受到这种影响并浪费资源。</p>

<p>复制每个库 API 以接受由每个设备特定分配器分配的输入或输出内存不是一个可行的选择。这将导致库入口点的组合爆炸。</p>

<p>随着高级语言结构（在 C++ 中，但在其他语言中）的进步，编译器现在可以在没有程序员知识的情况下利用 GPU 和其他设备。某些编译器识别的模式仅适用于共享地址空间。对所有其他模式使用共享地址空间也更合理。</p>

<h4 id="2-io总线与设备内存特性"><span class="me-2">2 IO总线与设备内存特性</span><a href="#2-io总线与设备内存特性" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>
<p><strong><a href="https://www.kernel.org/doc/html/latest/vm/hmm.html#id2">I/O 总线、设备内存特性</a></strong></p>

<p>io总线的访问存在缓存一致性问题，以及访问带宽问题。虽然已经有新的协议加入PCIE中，希望来解决前面问题。但并不是所有都支持。</p>

<p>为了使共享地址空间有意义，我们不仅必须允许设备访问任何内存，而且还必须允许任何内存在设备使用时迁移到设备内存（在发生时阻止 CPU 访问）。</p>

<h4 id="3-共享地址空间及迁移"><span class="me-2">3 共享地址空间及迁移</span><a href="#3-共享地址空间及迁移" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>
<p><strong><a href="https://www.kernel.org/doc/html/latest/vm/hmm.html#id3">共享地址空间和迁移</a></strong></p>

<p>Hmm提供两个功能：</p>

<p>1、统一虚拟地址空间：通过复制设备页表中的 CPU 页表来共享地址空间，因此对于进程地址空间中的任何有效主内存地址，相同的地址指向相同的物理内存；</p>

<p>2、ZONE_DEVICE 内存：它允许为设备内存的每个页面分配一个结构页面，管理设备内存。</p>

<h4 id="4-地址空间镜像实现"><span class="me-2">4 地址空间镜像实现</span><a href="#4-地址空间镜像实现" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>
<p><strong><a href="https://www.kernel.org/doc/html/latest/vm/hmm.html#id4">地址空间镜像实现和API</a></strong></p>

<p>1、页表同步：mmu_interval_notifier_insert</p>

<p>2、设备驱动程序想要填充一个虚拟地址范围：hmm_range_fault</p>

<p>例子：</p>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
</pre></td><td class="rouge-code"><pre> <span class="kt">int</span> <span class="nf">driver_populate_range</span><span class="p">(...)</span>
 <span class="p">{</span>
      <span class="k">struct</span> <span class="n">hmm_range</span> <span class="n">range</span><span class="p">;</span>
      <span class="p">...</span>

      <span class="n">range</span><span class="p">.</span><span class="n">notifier</span> <span class="o">=</span> <span class="o">&amp;</span><span class="n">interval_sub</span><span class="p">;</span>
      <span class="n">range</span><span class="p">.</span><span class="n">start</span> <span class="o">=</span> <span class="p">...;</span>
      <span class="n">range</span><span class="p">.</span><span class="n">end</span> <span class="o">=</span> <span class="p">...;</span>
      <span class="n">range</span><span class="p">.</span><span class="n">hmm_pfns</span> <span class="o">=</span> <span class="p">...;</span>

      <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="n">mmget_not_zero</span><span class="p">(</span><span class="n">interval_sub</span><span class="o">-&gt;</span><span class="n">notifier</span><span class="p">.</span><span class="n">mm</span><span class="p">))</span>
          <span class="k">return</span> <span class="o">-</span><span class="n">EFAULT</span><span class="p">;</span>

 <span class="nl">again:</span>
      <span class="n">range</span><span class="p">.</span><span class="n">notifier_seq</span> <span class="o">=</span> <span class="n">mmu_interval_read_begin</span><span class="p">(</span><span class="o">&amp;</span><span class="n">interval_sub</span><span class="p">);</span>
      <span class="n">mmap_read_lock</span><span class="p">(</span><span class="n">mm</span><span class="p">);</span>
      <span class="n">ret</span> <span class="o">=</span> <span class="n">hmm_range_fault</span><span class="p">(</span><span class="o">&amp;</span><span class="n">range</span><span class="p">);</span>
      <span class="k">if</span> <span class="p">(</span><span class="n">ret</span><span class="p">)</span> <span class="p">{</span>
          <span class="n">mmap_read_unlock</span><span class="p">(</span><span class="n">mm</span><span class="p">);</span>
          <span class="k">if</span> <span class="p">(</span><span class="n">ret</span> <span class="o">==</span> <span class="o">-</span><span class="n">EBUSY</span><span class="p">)</span>
                 <span class="k">goto</span> <span class="n">again</span><span class="p">;</span>
          <span class="k">return</span> <span class="n">ret</span><span class="p">;</span>
      <span class="p">}</span>
      <span class="n">mmap_read_unlock</span><span class="p">(</span><span class="n">mm</span><span class="p">);</span>

      <span class="n">take_lock</span><span class="p">(</span><span class="n">driver</span><span class="o">-&gt;</span><span class="n">update</span><span class="p">);</span>
      <span class="k">if</span> <span class="p">(</span><span class="n">mmu_interval_read_retry</span><span class="p">(</span><span class="o">&amp;</span><span class="n">ni</span><span class="p">,</span> <span class="n">range</span><span class="p">.</span><span class="n">notifier_seq</span><span class="p">)</span> <span class="p">{</span>
          <span class="n">release_lock</span><span class="p">(</span><span class="n">driver</span><span class="o">-&gt;</span><span class="n">update</span><span class="p">);</span>
          <span class="k">goto</span> <span class="n">again</span><span class="p">;</span>
      <span class="p">}</span>

      <span class="cm">/* Use pfns array content to update device page table,
       * under the update lock */</span>

      <span class="n">release_lock</span><span class="p">(</span><span class="n">driver</span><span class="o">-&gt;</span><span class="n">update</span><span class="p">);</span>
      <span class="k">return</span> <span class="mi">0</span><span class="p">;</span>
 <span class="p">}</span>
</pre></td></tr></tbody></table></code></div></div>

<h4 id="5-内存迁移"><span class="me-2">5 内存迁移</span><a href="#5-内存迁移" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>
<p><strong><a href="https://www.kernel.org/doc/html/latest/vm/hmm.html#id7">移入和移出设备内存</a></strong></p>

<p>由于 CPU 无法直接访问设备内存，因此设备驱动程序必须使用硬件 DMA 或设备特定的加载/存储指令来迁移数据。migrate_vma_setup()、migrate_vma_pages() 和 migrate_vma_finalize() 函数旨在使驱动程序更易于编写并集中跨驱动程序的通用代码。</p>

<h4 id="6-内存记账"><span class="me-2">6 内存记账</span><a href="#6-内存记账" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>
<p><strong><a href="https://www.kernel.org/doc/html/latest/vm/hmm.html#id8">内存 cgroup (memcg) 和 rss 记帐</a></strong></p>

<p>目前，设备内存被视为 rss 计数器中的任何常规页面（如果设备页面用于匿名，则为匿名，如果设备页面用于文件支持页面，则为文件，如果设备页面用于共享内存，则为 shmem）。这是为了保持现有应用程序的故意选择，这些应用程序可能在不知情的情况下开始使用设备内存，运行不受影响。</p>

<p>一个缺点是 OOM 杀手可能会杀死使用大量设备内存而不是大量常规系统内存的应用程序，因此不会释放太多系统内存。在决定以不同方式计算设备内存之前，我们希望收集更多关于应用程序和系统在存在设备内存的情况下在内存压力下如何反应的真实世界经验。</p>

<p>对内存 cgroup 做出了相同的决定。设备内存页面根据相同的内存 cgroup 计算，常规页面将被计算在内。这确实简化了进出设备内存的迁移。这也意味着从设备内存迁移回常规内存不会失败，因为它会超过内存 cgroup 限制。一旦我们对设备内存的使用方式及其对内存资源控制的影响有了更多的了解，我们可能会在后面重新考虑这个选择。</p>

<h3 id="二深入"><span class="me-2">二、深入</span><a href="#二深入" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h3>

<p>https://www.kernel.org/doc/html/latest/vm/hmm.html</p>

<div class="language-plaintext highlighter-rouge"><div class="code-header">
        <span data-label-text="Plaintext"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
</pre></td><td class="rouge-code"><pre>https://www.jianshu.com/p/7e2f9b8d59ca
HMM的讨论从社团的一个问题开始：像GPU这样的设备需要哪些功能以便能够支持HMM？ 答案是，它需要某种页表结构，可用于设置每页内存的访问权限。 例如，这取决于在相关页面中找到哪种类型的代码，设置权限使得其可以执行在CPU或GPU上（但不是在两者上）。 HMM还需要能够防止两个处理器的同时写入，因此GPU需要能够处理故障（handle faults）。
</pre></td></tr></tbody></table></code></div></div>

<h4 id="1-硬件功能"><span class="me-2">1 硬件功能</span><a href="#1-硬件功能" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>
<font color="red">(1)像GPU这样的设备需要哪些功能以便能够支持HMM？</font>

<font color="red">一致的设备内存节点CDM coherent device memory nodes？</font>

<div class="language-plaintext highlighter-rouge"><div class="code-header">
        <span data-label-text="Plaintext"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
</pre></td><td class="rouge-code"><pre>http://www.voidcn.com/article/p-gtgnskkx-bad.html
</pre></td></tr></tbody></table></code></div></div>
<p>异构内存管理（HMM）让设备的驱动程序可以为受制于自身内存管理的进程来镜像地址空间。正如Red Hat的开发人员杰尔姆·格利瑟（Jérôme Glisse）解释（https://lkml.org/lkml/2017/4/21/872），这项功能让GPU之类的硬件设备更容易直接访问进程的内存，没有复制任何内容所带来的额外开销。它也并不侵犯现代操作系统所提供的内存保护功能。</p>

<p>势必会从HMM获得最大好处的一类应用就是基于GPU的机器学习。诸如OpenCL和CUDA之类的机器学习库的速度到时有望得到HMM的提升。HMM做到这点的方式与针对基于GPU的机器学习进行的提速几乎一模一样，也就是说让数据留在原地、靠近GPU，在那里直接处理数据，然后传输尽可能少的数据。</p>

<p>让HMM适用于Linux中面临几个障碍。首先是内核支持，而在相当长的时间里，内核对HMM的支持一直情况不明朗。早在2014年，HMM最初是作为Linux内核补丁集（patchset）提出来的，Red Hat和英伟达都是关键开发商。涉及的工作量并不小，但是开发人员认为，可以提交代码，说不定可以加入到未来发布的几个内核版本。</p>

<p>第二个障碍是视频驱动程序支持，英伟达一致在单独搞这项工作。据格利瑟声称，AMD GPU可能也会支持HMM，所以这种特别的优化不会仅限于英伟达GPU。AMD一直在提升其在GPU市场的实力，因此可能会在同一块晶片上整合GPU处理和CPU处理（http://www.infoworld.com/article/3099204/hardware/amd-mulls-a-cpugpu-super-chip-in-a-server-reboot.html）。然而，软件生态系统仍然明显青睐英伟达；将来需要另外几个像HMM这样与厂商中立的项目，以及与CUDA所提供的性能不相上下的OpenCL性能，才可能获得切实的选择。</p>

<p>第三个障碍是硬件支持，因为HMM需要有一种可再现的页面故障（replayable page faults）硬件功能才能发挥作用。目前只有英伟达的Pascal高端GPU系列支持这项功能。从某种意义上来说，这是好消息，因为它意味着英伟达只需要为某一个硬件提供驱动程序支持――它需要完成的工作量较小，就可以让HMM正常使用起来。</p>

<p>一旦HMM实施到位，使用GPU实例的公共云提供商将面临压力：需要支持最新最好一代的GPU。不是仅仅把老式的英伟达Kepler显卡换成最先进的Pascal GPU就完事。由于后续的每一代GPU会显得更脱颖而出，像HMM这样的支持优化将带来战略优势。</p>

<h4 id="2-replayable-page-faults"><span class="me-2">2 replayable page faults</span><a href="#2-replayable-page-faults" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>
<font color="red">(2)硬件支持：replayable page faults硬件功能是什么？</font>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
</pre></td><td class="rouge-code"><pre><span class="nl">https:</span><span class="c1">//on-demand.gputechconf.com/gtc/2017/presentation/s7764_john-hubbardgpus-using-hmm-blur-the-lines-between-cpu-and-gpu.pdf</span>
</pre></td></tr></tbody></table></code></div></div>

<h4 id="3-how-hmm-works"><span class="me-2">3 How HMM works?</span><a href="#3-how-hmm-works" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>

<p><a href="/posts/./images/linux-hmm/hmm-works-1.png" class="popup img-link "><img data-src="/posts/./images/linux-hmm/hmm-works-1.png" style="zoom:0.5" class="lazyload" data-proofer-ignore></a></p>

<p><a href="/posts/./images/linux-hmm/hmm-works-2.png" class="popup img-link "><img data-src="/posts/./images/linux-hmm/hmm-works-2.png" style="zoom:0.5" class="lazyload" data-proofer-ignore></a></p>

<p><a href="/posts/./images/linux-hmm/hmm-works-3.png" class="popup img-link "><img data-src="/posts/./images/linux-hmm/hmm-works-3.png" style="zoom:0.5" class="lazyload" data-proofer-ignore></a></p>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
</pre></td><td class="rouge-code"><pre><span class="nl">https:</span><span class="c1">//nvidia.github.io/open-gpu-doc/manuals/turing/tu104/dev_mmu_fault.ref.txt</span>
<span class="mi">2</span>  <span class="o">-</span>  <span class="n">GPU</span> <span class="n">FAULT</span> <span class="n">BUFFER</span>
<span class="o">======================================</span>
<span class="n">This</span> <span class="n">chapter</span> <span class="n">describes</span> <span class="n">the</span> <span class="n">format</span> <span class="n">of</span> <span class="n">the</span> <span class="n">GPU</span> <span class="n">replayable</span> <span class="n">and</span>
<span class="n">non</span><span class="o">-</span><span class="n">replayable</span> <span class="n">fault</span> <span class="n">reporting</span> <span class="n">buffer</span> <span class="n">used</span> <span class="n">to</span> <span class="n">report</span> <span class="n">page</span> <span class="n">faults</span><span class="p">.</span>
    <span class="n">This</span> <span class="n">fault</span> <span class="n">buffer</span> <span class="n">is</span> <span class="n">written</span> <span class="n">to</span> <span class="n">by</span> <span class="n">GMMU</span> <span class="n">based</span> <span class="n">on</span> <span class="n">buffer</span> <span class="n">location</span> <span class="n">info</span>
<span class="n">set</span> <span class="n">in</span> <span class="n">GMMU</span> <span class="n">registers</span> <span class="p">(</span><span class="n">NV_PFB_PRI_MMU_REPLAY_FAULT_BUFFER_LO</span><span class="o">/</span><span class="n">HI</span> <span class="n">and</span>
<span class="n">NV_PFB_PRI_MMU_NON_REPLAY_FAULT_BUFFER_LO</span><span class="o">/</span><span class="n">HI</span><span class="p">).</span> <span class="n">The</span> <span class="n">replayable</span> <span class="n">fault</span>
<span class="n">buffer</span> <span class="n">is</span> <span class="n">managed</span> <span class="n">by</span> <span class="n">the</span> <span class="n">UVM</span> <span class="n">driver</span><span class="p">.</span> <span class="n">The</span> <span class="n">non</span><span class="o">-</span><span class="n">replayable</span> <span class="n">fault</span> <span class="n">buffer</span>
<span class="n">is</span> <span class="n">managed</span> <span class="n">by</span> <span class="n">RM</span><span class="p">.</span>
    <span class="n">The</span> <span class="n">buffer</span> <span class="n">can</span> <span class="n">overflow</span><span class="p">.</span> <span class="n">There</span> <span class="n">is</span> <span class="n">status</span> <span class="n">maintained</span> <span class="n">in</span> <span class="n">GMMU</span> <span class="k">register</span>
<span class="k">for</span> <span class="n">that</span><span class="p">.</span> <span class="n">If</span> <span class="n">the</span> <span class="n">buffer</span> <span class="n">has</span> <span class="n">overflowed</span> <span class="n">the</span> <span class="n">GPU</span> <span class="n">will</span> <span class="n">stop</span> <span class="n">writing</span> <span class="n">out</span> <span class="n">new</span> <span class="n">fault</span>
<span class="n">entries</span> <span class="n">and</span> <span class="n">proceed</span> <span class="n">to</span> <span class="n">drop</span> <span class="n">entries</span> <span class="n">until</span> <span class="n">SW</span> <span class="n">resets</span> <span class="n">the</span> <span class="n">overflow</span>
<span class="n">status</span> <span class="p">(</span><span class="n">normally</span> <span class="n">after</span> <span class="n">processing</span> <span class="n">the</span> <span class="n">existing</span> <span class="n">fault</span> <span class="n">packets</span> <span class="n">and</span> <span class="n">so</span>
<span class="n">GET</span> <span class="n">PTR</span> <span class="n">is</span> <span class="n">changed</span><span class="p">).</span> <span class="n">This</span> <span class="n">is</span> <span class="n">done</span> <span class="n">to</span> <span class="n">prevent</span> <span class="n">the</span> <span class="n">GPU</span> <span class="n">from</span> <span class="n">overwriting</span>
<span class="n">unprocessed</span> <span class="n">entries</span><span class="p">.</span>  <span class="n">When</span> <span class="n">faults</span> <span class="n">are</span> <span class="n">dropped</span> <span class="n">they</span> <span class="n">are</span> <span class="n">not</span> <span class="n">lost</span> <span class="k">for</span>
<span class="n">replayable</span> <span class="n">faults</span> <span class="n">as</span> <span class="n">the</span> <span class="n">requets</span> <span class="n">are</span> <span class="n">buffered</span> <span class="n">in</span> <span class="n">MMU</span> <span class="n">replay</span> <span class="n">buffer</span><span class="p">;</span>
<span class="n">however</span><span class="p">,</span> <span class="n">non</span><span class="o">-</span><span class="n">replayable</span> <span class="n">faults</span> <span class="n">are</span> <span class="n">lost</span> <span class="n">as</span> <span class="n">those</span> <span class="n">requests</span> <span class="n">are</span> <span class="n">not</span>
<span class="n">buffered</span> <span class="k">for</span> <span class="n">further</span> <span class="n">processing</span><span class="p">;</span> <span class="n">when</span> <span class="n">SW</span> <span class="n">triggers</span> <span class="n">a</span> <span class="n">replay</span> <span class="n">event</span> <span class="n">the</span>
<span class="n">requests</span> <span class="k">for</span> <span class="n">the</span> <span class="n">dropped</span> <span class="n">replayable</span> <span class="n">faults</span> <span class="n">will</span> <span class="n">be</span> <span class="n">replayed</span><span class="p">,</span> <span class="n">fault</span>
<span class="n">again</span><span class="p">,</span> <span class="n">and</span> <span class="n">then</span> <span class="n">be</span> <span class="n">reported</span> <span class="n">in</span> <span class="n">the</span> <span class="n">fault</span> <span class="n">buffer</span><span class="p">.</span>
</pre></td></tr></tbody></table></code></div></div>
<p>缓冲区可能溢出。GMMU寄存器中保存了这种状态。如果缓冲区溢出，GPU将停止写入新的故障条目，并继续丢弃条目，直到SW重置溢出状态（通常在处理现有的故障数据包之后，因此GET PTR被更改）。这样做是为了防止GPU覆盖未处理的条目。当故障丢弃时，它们不会因为可重放的错误而丢失，因为请求被缓存在MMU重放缓冲区中；但是，由于这些请求没有缓存以便进一步处理，非重放的错误会丢失；当SW触发重放事件时，对丢弃的可重放故障的请求将被重放，再次发生故障，然后在故障缓冲区中报告。</p>

<p>replayable page faults</p>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
</pre></td><td class="rouge-code"><pre><span class="nl">https:</span><span class="c1">//smartech.gatech.edu/bitstream/handle/1853/62741/KIM-DISSERTATION-2020.pdf</span>
</pre></td></tr></tbody></table></code></div></div>

<div class="language-plaintext highlighter-rouge"><div class="code-header">
        <span data-label-text="Plaintext"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
</pre></td><td class="rouge-code"><pre>https://www.redhat.com/files/summit/session-assets/2017/S104078-hubbard.pdf
</pre></td></tr></tbody></table></code></div></div>

<p><a href="/posts/./images/linux-hmm/redhat-hmm-introduction-1.png" class="popup img-link "><img data-src="/posts/./images/linux-hmm/redhat-hmm-introduction-1.png" style="zoom:0.8" class="lazyload" data-proofer-ignore></a></p>

<p><a href="/posts/./images/linux-hmm/redhat-hmm-introduction-2.png" class="popup img-link "><img data-src="/posts/./images/linux-hmm/redhat-hmm-introduction-2.png" style="zoom:0.6" class="lazyload" data-proofer-ignore></a></p>

<p><a href="/posts/./images/linux-hmm/redhat-hmm-introduction-3.png" class="popup img-link "><img data-src="/posts/./images/linux-hmm/redhat-hmm-introduction-3.png" style="zoom:0.6" class="lazyload" data-proofer-ignore></a></p>

<p><a href="/posts/./images/linux-hmm/redhat-hmm-introduction-4.png" class="popup img-link "><img data-src="/posts/./images/linux-hmm/redhat-hmm-introduction-4.png" style="zoom:0.6" class="lazyload" data-proofer-ignore></a></p>

<h3 id="三-疑问"><span class="me-2">三 疑问</span><a href="#三-疑问" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h3>

<p>(1)driver如何收到GPU运行的pagefault? 为什么gpu会产生page faults?  GPU有单独的MMU？</p>

<p>GPU page fault:</p>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
</pre></td><td class="rouge-code"><pre><span class="nl">https:</span><span class="c1">//winddoing.github.io/post/793.html</span>
<span class="n">GPU</span> <span class="err">页错误</span>
<span class="n">A</span> <span class="n">GPU</span> <span class="n">page</span> <span class="n">fault</span> <span class="n">commonly</span> <span class="n">occurs</span> <span class="n">under</span> <span class="n">one</span> <span class="n">of</span> <span class="n">these</span> <span class="n">conditions</span><span class="p">.</span> <span class="n">An</span> <span class="n">application</span> <span class="n">mistakenly</span> <span class="n">executes</span> <span class="n">work</span> <span class="n">on</span> <span class="n">the</span> <span class="n">GPU</span> <span class="n">that</span> <span class="n">references</span> <span class="n">a</span> <span class="n">deleted</span> <span class="n">object</span><span class="p">.</span> <span class="n">This</span> <span class="n">is</span> <span class="n">one</span> <span class="n">of</span> <span class="n">the</span> <span class="n">top</span> <span class="n">reasons</span> <span class="k">for</span> <span class="n">an</span> <span class="n">unexpected</span> <span class="n">device</span> <span class="n">removal</span><span class="p">.</span> <span class="n">An</span> <span class="n">application</span> <span class="n">mistakenly</span> <span class="n">executes</span> <span class="n">work</span> <span class="n">on</span> <span class="n">the</span> <span class="n">GPU</span> <span class="n">that</span> <span class="n">accesses</span> <span class="n">an</span> <span class="n">evicted</span> <span class="n">resource</span><span class="p">,</span> <span class="n">or</span> <span class="n">a</span> <span class="n">non</span><span class="o">-</span><span class="n">resident</span> <span class="n">tile</span><span class="p">.</span>
</pre></td></tr></tbody></table></code></div></div>
<p>GPU 页面错误通常在下列情况之一下发生：
1) 应用程序在 GPU 上错误地执行了应用已删除的对象的作业。 这是意外删除设备的主要原因之一。
2) 应用程序错误地在 GPU 上执行了访问已逐出的资源或非驻留磁贴的作业。
3) 着色器引用未初始化的或过时的描述符。
4) 着色器索引超出根绑定末尾。</p>
<div class="language-plaintext highlighter-rouge"><div class="code-header">
        <span data-label-text="Plaintext"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
3
4
</pre></td><td class="rouge-code"><pre>
```c
https://www.zhihu.com/question/381126048
链接：https://www.zhihu.com/question/381126048/answer/1093535046
</pre></td></tr></tbody></table></code></div></div>
<p>现在的CPU，是通过将需要GPU执行的命令写入内存（主存），然后通知GPU命令开始的地址与长度，来与GPU协作的。GPU可见的内存一般分为两部分：和CPU共享的（一般为主存的一部分），以及独占的（一般为显存）。GPU执行的程序以及相关参数（如顶点坐标），一般放在共享的内存里面，因为这些是需要CPU不断更新的。
GPU绘图所需资源（如贴图），由CPU从外存（如硬盘）读入之后，先临时放在共享内存上，然后通知GPU将其转移到GPU专有内存（显存）当中。现代GPU在绘图命令之外，一般还至少有一个专门的用来转移数据的命令队列，可以和绘图命令队列并行执行。GPU的执行结果（绘制的画面）一般直接写入显存当中的特定区域（屏幕缓冲区），然后由专门的ScanOut电路（包括在GPU当中或者显卡上）对其完成最终的数字信号处理之后（如色空间转换，Gamma调整等）输出给显示设备。因此，如果CPU想要读取GPU计算的结果，需要通知GPU将结果传输回共享内存区域，这通常有较大的性能开销。</p>

<div class="language-c highlighter-rouge"><div class="code-header">
        <span data-label-text="C"><i class="fas fa-code fa-fw small"></i></span>
      <button aria-label="copy" data-title-succeed="Copied!"><i class="far fa-clipboard"></i></button></div><div class="highlight"><code><table class="rouge-table"><tbody><tr><td class="rouge-gutter gl"><pre class="lineno">1
2
</pre></td><td class="rouge-code"><pre><span class="nl">http:</span><span class="c1">//www.irisa.fr/alf/downloads/ADA/Tanasic_ExceptionGPUs_Micro17.pdf</span>
<span class="n">Ecient</span> <span class="n">Exception</span> <span class="n">Handling</span> <span class="n">Support</span> <span class="k">for</span> <span class="n">GPUs</span>
</pre></td></tr></tbody></table></code></div></div>

<p>GPU有独立的MMU。</p>

<p>(2)CPU如何识别是设备内存的page fault？</p>

<p>中断通知driver， 处理faults.</p>

<h3 id="四总结支持hmm的要求"><span class="me-2">四、总结支持HMM的要求</span><a href="#四总结支持hmm的要求" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h3>

<p>（1）设备支持handles faults能力，支持类似MMU页表能力</p>

<p>​	 通过CPU和device的页表设置来保证设备和CPU只有一个在访问虚拟地址</p>

<p>（2）设备支持replay faults的；</p>

<p>​		驱动一般通过中断通知driver处理faults, driver需要存储replayed fault(page faults)，driver处理完fault后需要replay触发faults的语句。</p>

<h3 id="五hmm-vs-dsm"><span class="me-2">五、HMM VS DSM</span><a href="#五hmm-vs-dsm" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h3>
<h4 id="1-hmm"><span class="me-2">1 HMM</span><a href="#1-hmm" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>

<p>（1）设备支持handles faults能力，支持类似MMU页表能力</p>

<p>（2）设备支持replay faults的</p>

<h4 id="2-dsm"><span class="me-2">2 DSM</span><a href="#2-dsm" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>

<p>(1) 由于线程共享相同的虚拟地址空间，因此所有线程必须具有相同的内存视图。</p>

<h4 id="3-hmm---vs---dsm"><span class="me-2">3 HMM   VS   DSM</span><a href="#3-hmm---vs---dsm" class="anchor text-muted"><i class="fas fa-hashtag"></i></a></h4>

<div class="table-wrapper"><table>
  <thead>
    <tr>
      <th> </th>
      <th>前置条件</th>
      <th>优势</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>HMM</td>
      <td>(1) 支持handles faults，支持类似MMU页表能力，可以设置not present含义<br />(2) 支持handles faults之后replay fault的方法。</td>
      <td> </td>
    </tr>
    <tr>
      <td>DSM</td>
      <td>(1) 所有va都是相同的内存视图<br />Main memory需要预留和device memory完全镜像的内存。<br />要支持相同的地址翻译硬件设备。<br />(2) page coherency机制需要消息通知或者监控手段。</td>
      <td>1、数据没有发生写，不需要同步，可以直接访问。</td>
    </tr>
  </tbody>
</table></div>


</div>

<div class="post-tail-wrapper text-muted">

  <!-- categories -->
  
  <div class="post-meta mb-3">
    <i class="far fa-folder-open fa-fw me-1"></i>
    
      <a href='/categories/linux/'>linux</a>,
      <a href='/categories/hmm/'>HMM</a>
  </div>
  
  <!-- post.html -->

  <!-- tags -->
  
  <div class="post-tags">
    <i class="fa fa-tags fa-fw me-1"></i>
      
      <a href="/tags/hmm/"
          class="post-tag no-text-decoration" >HMM</a>
      
  </div>
  
  
  <div class="post-tail-bottom
    d-flex justify-content-between align-items-center mt-3 pt-5 pb-2">
    <div class="license-wrapper">

      

        

        This post is licensed under 
        <a href="https://creativecommons.org/licenses/by/4.0/">
          CC BY 4.0
        </a>
         by the author.

      
    </div>

    <!-- Post sharing snippet -->

<div class="share-wrapper">
  <span class="share-label text-muted me-1">Share</span>
  <span class="share-icons">
    
    
    

    
      
      <a
        href="https://twitter.com/intent/tweet?text=Linux%20Heterogeneous%20memory%20management%20(HMM)%20-%20Cheng%20Luo&url=https%3A%2F%2Fluochenglcs.github.io%2Fposts%2Fkernel-hmm%2F"
        data-bs-toggle="tooltip"
        data-bs-placement="top"
        title="Twitter"
        target="_blank"
        rel="noopener"
        aria-label="Twitter"
      >
        <i class="fa-fw fab fa-twitter"></i>
      </a>
    
      
      <a
        href="https://www.facebook.com/sharer/sharer.php?title=Linux%20Heterogeneous%20memory%20management%20(HMM)%20-%20Cheng%20Luo&u=https%3A%2F%2Fluochenglcs.github.io%2Fposts%2Fkernel-hmm%2F"
        data-bs-toggle="tooltip"
        data-bs-placement="top"
        title="Facebook"
        target="_blank"
        rel="noopener"
        aria-label="Facebook"
      >
        <i class="fa-fw fab fa-facebook-square"></i>
      </a>
    
      
      <a
        href="https://t.me/share/url?url=https%3A%2F%2Fluochenglcs.github.io%2Fposts%2Fkernel-hmm%2F&text=Linux%20Heterogeneous%20memory%20management%20(HMM)%20-%20Cheng%20Luo"
        data-bs-toggle="tooltip"
        data-bs-placement="top"
        title="Telegram"
        target="_blank"
        rel="noopener"
        aria-label="Telegram"
      >
        <i class="fa-fw fab fa-telegram"></i>
      </a>
    

    <i
      id="copy-link"
      class="fa-fw fas fa-link small"
      data-bs-toggle="tooltip"
      data-bs-placement="top"
      title="Copy link"
      data-title-succeed="Link copied successfully!"
    >
    </i>
  </span>
</div>


  </div><!-- .post-tail-bottom -->

</div><!-- div.post-tail-wrapper -->


      
    
      
    </div>
  </div>
  <!-- #core-wrapper -->

  <!-- panel -->
  <div id="panel-wrapper" class="col-xl-3 ps-2 text-muted">
    <div class="access">
      <!-- Get the last 5 posts from lastmod list. -->














  <div id="access-lastmod" class="post">
    <div class="panel-heading">Recently Updated</div>
    <ul class="post-content list-unstyled ps-0 pb-1 ms-1 mt-2">
      
        
        
        
        <li class="text-truncate lh-lg">
          <a href="/posts/%E5%86%85%E5%AD%98%E6%95%B0%E6%8D%AE%E5%BA%93/">MMDB (memory database)</a>
        </li>
      
        
        
        
        <li class="text-truncate lh-lg">
          <a href="/posts/userspace-rcu%E9%94%81/">userspace rcu lock</a>
        </li>
      
        
        
        
        <li class="text-truncate lh-lg">
          <a href="/posts/Real-time-operating-system/">Realtime Operating System</a>
        </li>
      
        
        
        
        <li class="text-truncate lh-lg">
          <a href="/posts/HUNDSUN-LightOS/">Hundsun Lightos</a>
        </li>
      
        
        
        
        <li class="text-truncate lh-lg">
          <a href="/posts/linux-struct-folio/">struct folio</a>
        </li>
      
    </ul>
  </div>
  <!-- #access-lastmod -->


      <!-- The trending tags list -->















  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
        
        

  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
        
        

  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        



  <div id="access-tags">
    <div class="panel-heading">Trending Tags</div>
    <div class="d-flex flex-wrap mt-3 mb-1 me-3">
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/lock/">lock</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/libumem/">libumem</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/mm/">mm</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/autonuma/">autonuma</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/bfs/">BFS</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/bus/">Bus</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/caladan/">caladan</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/damon/">damon</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/deep-learning/">Deep Learning</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/dma/">DMA</a>
      
    </div>
  </div>


    </div>

    
      
      



  <div id="toc-wrapper" class="ps-0 pe-4 mb-5">
    <div class="panel-heading ps-3 pt-2 mb-2">Contents</div>
    <nav id="toc"><ul><li>Linux Heterogeneous memory management (HMM)<ul><li><a href="#一hmm是什么">一、HMM是什么？</a><ul><li><a href="#1-特定设备内存分配器">1 特定设备内存分配器</a></li><li><a href="#2-io总线与设备内存特性">2 IO总线与设备内存特性</a></li><li><a href="#3-共享地址空间及迁移">3 共享地址空间及迁移</a></li><li><a href="#4-地址空间镜像实现">4 地址空间镜像实现</a></li><li><a href="#5-内存迁移">5 内存迁移</a></li><li><a href="#6-内存记账">6 内存记账</a></li></ul></li><li><a href="#二深入">二、深入</a><ul><li><a href="#1-硬件功能">1 硬件功能</a></li><li><a href="#2-replayable-page-faults">2 replayable page faults</a></li><li><a href="#3-how-hmm-works">3 How HMM works?</a></li></ul></li><li><a href="#三-疑问">三 疑问</a></li><li><a href="#四总结支持hmm的要求">四、总结支持HMM的要求</a></li><li><a href="#五hmm-vs-dsm">五、HMM VS DSM</a><ul><li><a href="#1-hmm">1 HMM</a></li><li><a href="#2-dsm">2 DSM</a></li><li><a href="#3-hmm---vs---dsm">3 HMM   VS   DSM</a></li></ul></li></ul></li></ul></li></ul></nav>
  </div>
  


    
  </div>
</div>

<!-- tail -->

  <div class="row">
    <div id="tail-wrapper" class="col-12 col-lg-11 col-xl-9 px-3 pe-xl-4 mt-5">
      
        
        <!--
  Recommend the other 3 posts according to the tags and categories of the current post,
  if the number is not enough, use the other latest posts to supplement.
-->

<!-- The total size of related posts -->


<!-- An random integer that bigger than 0 -->


<!-- Equals to TAG_SCORE / {max_categories_hierarchy} -->








  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  
    
  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  

  






<!-- Fill with the other newlest posts -->





  <div id="related-posts" class="mb-2 mb-sm-4">
    <h3 class="pt-2 mb-4 ms-1" data-toc-skip>
      Further Reading
    </h3>
    <div class="row row-cols-1 row-cols-md-2 row-cols-xl-3 g-4 mb-4">
      
        
        
        <div class="col">
          <a href="/posts/linux-kcsan/" class="card post-preview h-100">
            <div class="card-body">
              <!--
  Date format snippet
  See: ${JS_ROOT}/utils/locale-dateime.js
-->





<em
  class="small"
  data-ts="1599782400"
  data-df="ll"
  
>
  Sep 11, 2020
</em>

              <h4 class="pt-0 my-2" data-toc-skip>The Kernel Concurrency Sanitizer (KCSAN)</h4>
              <div class="text-muted small">
                <p>
                  





                  1 kcasan

2 使用

                </p>
              </div>
            </div>
          </a>
        </div>
      
        
        
        <div class="col">
          <a href="/posts/linux-kprobe/" class="card post-preview h-100">
            <div class="card-body">
              <!--
  Date format snippet
  See: ${JS_ROOT}/utils/locale-dateime.js
-->





<em
  class="small"
  data-ts="1600041600"
  data-df="ll"
  
>
  Sep 14, 2020
</em>

              <h4 class="pt-0 my-2" data-toc-skip>Kernel Probe (kprobe)</h4>
              <div class="text-muted small">
                <p>
                  





                  1 kprobe

2 debugfs kprobe trace
kprobe trace 使用debugfs的trace功能
 p[:[GRP/]EVENT] [MOD:]SYM[+offs]|MEMADDR [FETCHARGS]  : Set a probe
 r[MAXACTIVE][:[GRP/]EVENT] [MOD:]SYM[+0] [FETCHARGS]  : Set a r...
                </p>
              </div>
            </div>
          </a>
        </div>
      
        
        
        <div class="col">
          <a href="/posts/linux-userfaultfd/" class="card post-preview h-100">
            <div class="card-body">
              <!--
  Date format snippet
  See: ${JS_ROOT}/utils/locale-dateime.js
-->





<em
  class="small"
  data-ts="1600905600"
  data-df="ll"
  
>
  Sep 24, 2020
</em>

              <h4 class="pt-0 my-2" data-toc-skip>Kernel userfaultfd</h4>
              <div class="text-muted small">
                <p>
                  





                  Userfault只支持匿名页，hugetlb、共享内存；

一、软件流程

1 初始化
调用__NR_userfaultfd syscall初始化
调用syscall初始化建立匿名inode文件，并初始化file-&amp;gt;private_data,并返回用户态文件fd。


  
    用户态：

    uffd = syscall(__NR_userfaultfd, O_CLOEXE...
                </p>
              </div>
            </div>
          </a>
        </div>
      
    </div>
    <!-- .card-deck -->
  </div>
  <!-- #related-posts -->


      
        
        <!-- Navigation buttons at the bottom of the post. -->

<div class="post-navigation d-flex justify-content-between">
  
    <a
      href="/posts/linux-ksm/"
      class="btn btn-outline-primary"
      prompt="Older"
    >
      <p>Kernel Samepage Merging (KSM)</p>
    </a>
  

  
    <a
      href="/posts/libumem-Introduce/"
      class="btn btn-outline-primary"
      prompt="Newer"
    >
      <p>libumem Introduce</p>
    </a>
  
</div>

      
        
        <!--  The comments switcher -->


      
    </div>
  </div>


        <!-- The Search results -->

<div id="search-result-wrapper" class="d-flex justify-content-center unloaded">
  <div class="col-11 post-content">
    <div id="search-hints">
      <!-- The trending tags list -->















  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
        
        

  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
        
        

  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        

  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
    
  
    
    
    
    
      
        
        



  <div id="access-tags">
    <div class="panel-heading">Trending Tags</div>
    <div class="d-flex flex-wrap mt-3 mb-1 me-3">
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/lock/">lock</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/libumem/">libumem</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/mm/">mm</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/autonuma/">autonuma</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/bfs/">BFS</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/bus/">Bus</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/caladan/">caladan</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/damon/">damon</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/deep-learning/">Deep Learning</a>
      
        
        <a class="post-tag btn btn-outline-primary" href="/tags/dma/">DMA</a>
      
    </div>
  </div>


    </div>
    <div id="search-results" class="d-flex flex-wrap justify-content-center text-muted mt-3"></div>
  </div>
</div>

      </div>
    </div>

    <!-- The Footer -->

<footer>
  <div class="container px-lg-4">
    <div class="d-flex justify-content-center align-items-center text-muted mx-md-3">
      <p>Using the <a href="https://jekyllrb.com" target="_blank" rel="noopener">Jekyll</a> theme <a href="https://github.com/cotes2020/jekyll-theme-chirpy" target="_blank" rel="noopener">Chirpy</a>
      </p>

      <p>©
        2023
        <a href="https://gitee.com/luochenglcs">luochunsheng</a>.
        
          <span
            data-bs-toggle="tooltip"
            data-bs-placement="top"
            title="Except where otherwise noted, the blog posts on this site are licensed under the Creative Commons Attribution 4.0 International (CC BY 4.0) License by the author."
          >Some rights reserved.</span>
        
      </p>
    </div>
  </div>
</footer>


    <div id="mask"></div>

    <button id="back-to-top" aria-label="back-to-top" class="btn btn-lg btn-box-shadow">
      <i class="fas fa-angle-up"></i>
    </button>

    
      <div
        id="notification"
        class="toast"
        role="alert"
        aria-live="assertive"
        aria-atomic="true"
        data-bs-animation="true"
        data-bs-autohide="false"
      >
        <div class="toast-header">
          <button
            type="button"
            class="btn-close ms-auto"
            data-bs-dismiss="toast"
            aria-label="Close"
          ></button>
        </div>
        <div class="toast-body text-center pt-0">
          <p class="px-2 mb-3">A new version of content is available.</p>
          <button type="button" class="btn btn-primary" aria-label="Update">
            Update
          </button>
        </div>
      </div>
    

    <!-- JS selector for site. -->

<!-- commons -->



<!-- layout specified -->


  

  
    <!-- image lazy-loading & popup & clipboard -->
    
  















  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  

  
    

    

  



  <script src="https://cdn.jsdelivr.net/combine/npm/jquery@3.7.0/dist/jquery.min.js,npm/bootstrap@5.2.3/dist/js/bootstrap.bundle.min.js,npm/simple-jekyll-search@1.10.0/dest/simple-jekyll-search.min.js,npm/lazysizes@5.3.2/lazysizes.min.js,npm/magnific-popup@1.1.0/dist/jquery.magnific-popup.min.js,npm/clipboard@2.0.11/dist/clipboard.min.js,npm/dayjs@1.11.7/dayjs.min.js,npm/dayjs@1.11.7/locale/en.min.js,npm/dayjs@1.11.7/plugin/relativeTime.min.js,npm/dayjs@1.11.7/plugin/localizedFormat.min.js,npm/tocbot@4.21.0/dist/tocbot.min.js"></script>






<script defer src="/assets/js/dist/post.min.js"></script>






    

    <!--
  Jekyll Simple Search loader
  See: <https://github.com/christian-fei/Simple-Jekyll-Search>
-->





<script>
  /* Note: dependent library will be loaded in `js-selector.html` */
  SimpleJekyllSearch({
    searchInput: document.getElementById('search-input'),
    resultsContainer: document.getElementById('search-results'),
    json: '/assets/js/data/search.json',
    searchResultTemplate: '<div class="px-1 px-sm-2 px-lg-4 px-xl-0">  <a href="{url}">{title}</a>  <div class="post-meta d-flex flex-column flex-sm-row text-muted mt-1 mb-1">    {categories}    {tags}  </div>  <p>{snippet}</p></div>',
    noResultsText: '<p class="mt-5"></p>',
    templateMiddleware: function(prop, value, template) {
      if (prop === 'categories') {
        if (value === '') {
          return `${value}`;
        } else {
          return `<div class="me-sm-4"><i class="far fa-folder fa-fw"></i>${value}</div>`;
        }
      }

      if (prop === 'tags') {
        if (value === '') {
          return `${value}`;
        } else {
          return `<div><i class="fa fa-tag fa-fw"></i>${value}</div>`;
        }
      }
    }
  });
</script>

  </body>
</html>

